# -*- coding:utf-8 -*-
import os
import sys
from lxml import etree

# use Django models
BASEDIR = os.path.dirname(os.path.realpath(__file__))
DB_BASEDIR = os.path.join(BASEDIR, 'db')
sys.path.insert(0, DB_BASEDIR)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "db.settings")
import django
django.setup()
from store import models

def log(text):
    print(text, file=sys.stderr)


class Spider:

    op = "save"

    def __init__(self, url):
        if 'KYO_LAGOU_OP' in os.environ:
            self.op = os.environ['KYO_LAGOU_OP']

        record = self.get_one(url)

        if self.op == "save":
            exit(self.save(record))
        else:
            print(record)

    def id_from_url(self, url):
        x = url.split('/')[-1]
        x = x.split('.')[0]
        return x

    def get_one(self, url):
        job_id = self.id_from_url(url)

        if self.op == "save"    \
                and models.LagouJob.objects.filter(job_id__exact=job_id):
            log('failed to exists job %s' % job_id)
            return None

        try:
            text = self.fetch(url)
        except AssertionError:
            log('failed to download job %s' % job_id)
            return None

        try:
            record = self.parse(text)
        except AssertionError:
            log('html source code error %s' % job_id)
            return None
        else:
            record['job_id'] = job_id
            return record

    def fetch(self, url):
        fp = open(url, 'r')
        text = fp.read()
        fp.close()
        return text

    def get_first(self, tree, xpath):
        es = tree.xpath(xpath)
        assert es, "not found"
        return es[0]

    def parse(self, text):
        tree = etree.HTML(text)

        # job title
        xpath = '//div[@class="job-name"]/span[@class="name"]'
        job_title = self.get_first(tree, xpath).text
        #  print(job_title)

        # job salary, experence, education
        xpath = '//dd[@class="job_request"]//span/text()'
        info = [x.replace('/', '').strip() for x in tree.xpath(xpath)]
        job_salary, x, job_experence, job_education, x = info

        # job tags
        xpath = '//dd[@class="job_request"]//li/text()'
        job_tags = ':'.join(tree.xpath(xpath))

        # job advantage
        xpath = '//dd[@class="job-advantage"]/p/text()'
        job_advantage = self.get_first(tree, xpath)

        # job detail
        xpath = '//dd[@class="job_bt"]//p/text()'
        info = [x.replace('\xa0', '') for x in tree.xpath(xpath)]
        job_detail = '\n'.join(info)

        # job address
        xpath = '//div[@class="work_addr"]//text()'
        info = tree.xpath(xpath)
        info = ' '.join(info).split()[:-1]
        info = [x for x in info if x != '-']
        job_address = ''.join(info)

        # pub date
        xpath = '//p[@class="publish_time"]/text()'
        pub_date = self.get_first(tree, xpath).split('\xa0')[0]

        # company id
        xpath = '//dl[@class="job_company"]/dt/a'
        a = self.get_first(tree, xpath)
        url = a.attrib['href']
        company_id = self.id_from_url(url)

        # company name
        xpath = '//dl[@class="job_company"]/dt/a/img'
        x = self.get_first(tree, xpath)
        company_name = x.attrib['alt']

        # create time
        create_time = django.utils.timezone.now()

        record = {}
        record['job_title'] = job_title
        record['job_salary'] = job_salary
        record['job_experence'] = job_experence
        record['job_education'] = job_education
        record['job_tags'] = job_tags
        record['job_advantage'] = job_advantage
        record['job_detail'] = job_detail
        record['job_address'] = job_address
        record['company_id'] = company_id
        record['company_name'] = company_name
        record['pub_date'] = pub_date
        record['create_time'] = create_time

        return record

    def save(self, record):
        if not record:
            return 1
        m = models.LagouJob()
        m.job_id = record['job_id']
        m.job_title = record['job_title']
        m.job_salary = record['job_salary']
        m.job_experence = record['job_experence']
        m.job_education = record['job_education']
        m.job_tags = record['job_tags']
        m.job_advantage = record['job_advantage']
        m.job_detail = record['job_detail']
        m.job_address = record['job_address']
        m.pub_date = record['pub_date']
        m.company_id = record['company_id']
        m.company_name = record['company_name']
        m.create_time = record['create_time']
        m.save()

        return 0


if __name__ == '__main__':
    Spider(sys.argv[1])

